#! /usr/bin/python
# -*- coding: utf-8 -*-

import string, re

def plain(in_s, res):    
    """ string in utf-8 convert to ascii    
   
    In order to speed up the conversion, only those twins (pairs) should be matched
    that are really in real world data. 

    This is the one possible demonstration how to find out whether pair is used at all.
    
    TODO:   Now we will know the counts, however it is hard as it is now 
            to map them back to their hex representation.
    """
    # dlzne, mekcene, vokane, dvojbodky, kruzky
    # Ä,ä,Á,á,Č,č,Ď,ď,É,é,Ě,ě,E,e,Í,í,Ĺ,ĺ,Ľ,ľ,Ń,ń,Ň,ň,N,n,Ó,ó,Ô,ô,Ŕ,ŕ,Ř,ř,Ś,ś,ß,Š,š,Ť,ť,Ü,ü,Ú,ú,Ů,ů,Ý,ý,Ź,ź,Ž,ž
    # 35MB txt file takes 1-2 minutes
    # character map source http://www.utf8-chartable.de/
    if 1:
        twins = {'\xc3\x84':'A','\xc4\x95':'e',	'\xc3\x93':'O',	'\xc5\xa4':'T',
                '\xc3\xa4':'a',	'\xc3\x8d':'I',	'\xc3\xb3':'o',	'\xc5\xa5':'t',
                '\xc3\x81':'A',	'\xc3\xad':'i',	'\xc3\x94':'O',	'\xc3\x9c':'U',
                '\xc3\xa1':'a',	'\xc4\xb9':'L',	'\xc3\xb4':'o',	'\xc3\xbc':'u',
                '\xc4\x8c':'C',	'\xc4\xba':'l',	'\xc5\x94':'R',	'\xc3\x9a':'U',
                '\xc4\x8d':'c',	'\xc4\xbd':'L',	'\xc5\x95':'r',	'\xc3\xba':'u',
                '\xc4\x8e':'D',	'\xc4\xbe':'l',	'\xc5\x98':'R',	'\xc5\xae':'U',
                '\xc4\x8f':'d',	'\xc5\x83':'N',	'\xc5\x99':'r',	'\xc5\xaf':'u',
                '\xc3\x89':'E',	'\xc5\x84':'n',	'\xc5\x9a':'S',	'\xc3\x9d':'Y',
                '\xc3\xa9':'e',	'\xc5\x87':'N',	'\xc5\x9b':'s',	'\xc3\xbd':'y',
                '\xc4\x9a':'E',	'\xc5\x88':'n',	'\xc3\x9f':'ss','\xc5\xb9':'Z',
                '\xc4\x9b':'e',	'\xc3\x91':'N',	'\xc5\xa0':'S',	'\xc5\xba':'z',
                '\xc4\x94':'E',	'\xc3\xb1':'n',	'\xc5\xa1':'s',	'\xc5\xbd':'Z',
                '\xc5\xbe':'z'
                }
    # Full UTF-8 Character Map ("word" characters only)
    # 35MB txt file takes approx. 40 minutes
    if 0:
        twins = {
                '\xc3\x80':'A', '\xc3\xb2':'o', '\xc4\xa1':'g','\xc5\x93':'ce',
                '\xc3\x81':'A',	'\xc3\xb3':'o',	'\xc4\xa2':'G',	'\xc5\x94':'R',
                '\xc3\x82':'A',	'\xc3\xb4':'o',	'\xc4\xa3':'g',	'\xc5\x95':'r',
                '\xc3\x83':'A',	'\xc3\xb5':'o',	'\xc4\xa4':'H',	'\xc5\x96':'R',
                '\xc3\x84':'A',	'\xc3\xb6':'o',	'\xc4\xa5':'h',	'\xc5\x97':'r',
                '\xc3\x85':'A',	'\xc3\xb7':'',	'\xc4\xa6':'H',	'\xc5\x98':'R',
                '\xc3\x86':'AE','\xc3\xb8':'o',	'\xc4\xa7':'h',	'\xc5\x99':'r',
                '\xc3\x87':'C',	'\xc3\xb9':'u',	'\xc4\xa8':'I',	'\xc5\x9a':'S',
                '\xc3\x88':'E',	'\xc3\xba':'u',	'\xc4\xa9':'i',	'\xc5\x9b':'s',
                '\xc3\x89':'E',	'\xc3\xbb':'u',	'\xc4\xaa':'I',	'\xc5\x9c':'S',
                '\xc3\x8a':'E',	'\xc3\xbc':'u',	'\xc4\xab':'i',	'\xc5\x9d':'s',
                '\xc3\x8b':'E',	'\xc3\xbd':'y',	'\xc4\xac':'I',	'\xc5\x9e':'S',
                '\xc3\x8c':'I',	'\xc3\xbe':'',	'\xc4\xad':'i',	'\xc5\x9f':'s',
                '\xc3\x8d':'I',	'\xc3\xbf':'y',	'\xc4\xae':'I',	'\xc5\xa0':'S',
                '\xc3\x8e':'I',	'\xc3\x9e':'P',	'\xc4\xaf':'i',	'\xc5\xa1':'s',
                '\xc3\x8f':'I',	'\xc3\xa6':'ae','\xc4\xb0':'I',	'\xc5\xa2':'T',
                '\xc3\x90':'D',	'\xc3\xb0':'d',	'\xc4\xb1':'i',	'\xc5\xa3':'t',
                '\xc3\x91':'N',	'\xc4\x80':'A',	'\xc4\xb2':'IJ','\xc5\xa4':'T',
                '\xc3\x92':'O',	'\xc4\x81':'a',	'\xc4\xb3':'ij','\xc5\xa5':'t',
                '\xc3\x93':'O',	'\xc4\x82':'A',	'\xc4\xb4':'J',	'\xc5\xa6':'T',
                '\xc3\x94':'O',	'\xc4\x83':'a',	'\xc4\xb5':'J',	'\xc5\xa7':'t',
                '\xc3\x95':'O',	'\xc4\x84':'A',	'\xc4\xb6':'K',	'\xc5\xa8':'U',
                '\xc3\x96':'O',	'\xc4\x85':'a',	'\xc4\xb7':'k',	'\xc5\xa9':'u',
                '\xc3\x97':'',	'\xc4\x86':'C',	'\xc4\xb8':'K',	'\xc5\xaa':'U',
                '\xc3\x98':'O',	'\xc4\x87':'c',	'\xc4\xb9':'L',	'\xc5\xab':'u',
                '\xc3\x99':'U',	'\xc4\x88':'C',	'\xc4\xba':'l',	'\xc5\xac':'U',
                '\xc3\x9a':'U',	'\xc4\x89':'c',	'\xc4\xbb':'L',	'\xc5\xad':'u',
                '\xc3\x9b':'U',	'\xc4\x8a':'C',	'\xc4\xbc':'l',	'\xc5\xae':'U',
                '\xc3\x9c':'U',	'\xc4\x8b':'c',	'\xc4\xbd':'L',	'\xc5\xaf':'u',
                '\xc3\x9d':'Y',	'\xc4\x8c':'C',	'\xc4\xbe':'l',	'\xc5\xb0':'U',
                '\xc3\x9e':'',	'\xc4\x8d':'c',	'\xc4\xbf':'L',	'\xc5\xb1':'u',
                '\xc3\x9f':'ss','\xc4\x8e':'D',	'\xc5\x80':'l',	'\xc5\xb2':'U',
                '\xc3\xa0':'a',	'\xc4\x8f':'d',	'\xc5\x81':'L',	'\xc5\xb3':'u',
                '\xc3\xa1':'a',	'\xc4\x90':'D',	'\xc5\x82':'l',	'\xc5\xb4':'W',
                '\xc3\xa2':'a',	'\xc4\x91':'d',	'\xc5\x83':'N',	'\xc5\xb5':'w',
                '\xc3\xa3':'a',	'\xc4\x92':'E',	'\xc5\x84':'n',	'\xc5\xb6':'Y',
                '\xc3\xa4':'a',	'\xc4\x93':'e',	'\xc5\x85':'N',	'\xc5\xb7':'y',
                '\xc3\xa5':'a',	'\xc4\x94':'E',	'\xc5\x86':'n',	'\xc5\xb8':'Y',
                '\xc3\xa6':'',	'\xc4\x95':'e',	'\xc5\x87':'N',	'\xc5\xb9':'Z',
                '\xc3\xa7':'c',	'\xc4\x96':'E',	'\xc5\x88':'n',	'\xc5\xba':'z',
                '\xc3\xa8':'e',	'\xc4\x97':'e',	'\xc5\x89':'n',	'\xc5\xbb':'Z',
                '\xc3\xa9':'e',	'\xc4\x98':'E',	'\xc5\x8a':'n',	'\xc5\xbc':'z',
                '\xc3\xaa':'e',	'\xc4\x99':'e',	'\xc5\x8b':'n',	'\xc5\xbd':'Z',
                '\xc3\xab':'e',	'\xc4\x9a':'E',	'\xc5\x8c':'O',	'\xc5\xbe':'z',
                '\xc3\xac':'i',	'\xc4\x9b':'e',	'\xc5\x8d':'o',	'\xc5\xbf':'s',
                '\xc3\xad':'i',	'\xc4\x9c':'G',	'\xc5\x8e':'O',	
                '\xc3\xae':'i',	'\xc4\x9d':'g',	'\xc5\x8f':'o',	
                '\xc3\xaf':'i',	'\xc4\x9e':'G',	'\xc5\x90':'O',	
                '\xc3\xb0':'',	'\xc4\x9f':'g',	'\xc5\x91':'o',	
                '\xc3\xb1':'n',	'\xc4\xa0':'G',	'\xc5\x92':'CE'
                }

    #res = {}
    #print res
    for k in sorted(twins.keys()):
        line0 = in_s
        in_s = re.sub(k,twins[k], in_s)
        line1 = in_s
        if line0 <> line1:
            if res.has_key(k):
                res[k] += 1
            else:
                res[k] = 1
   
    #print res
 
    return in_s, res

if __name__ == "__main__":
    #main()

    of = open('utf8sample.txt','r')
    lines = of.readlines()
    of.close()
    
    res = {}
    res_new = {}
    for line in lines:
        res = res_new
        clean_line, res_new = plain(line, res)
    
    for ks in sorted(res.keys()):
        print ks, res[ks]

